from SaveImage import get_driver, save_news_list, IMG_BASE
from requests import get, post
from common import *
import json

url = 'http://www.cnsoftbei.com/plus/list.php?tid=1'
xpath = '/html/body/div[11]/div/div[1]/div[2]'
CATE = 52

def get_info_list(url):
    from selenium.webdriver.common.by import By
    from bs4 import BeautifulSoup
    from random import randint
    import time

    driver = get_driver()

    driver.get(url)
    time.sleep(1)

    element = driver.find_elements(By.CLASS_NAME, 'plist4_h2')

    ret_list = []

    for a in element:
        a = a.find_element(By.TAG_NAME, 'a')
        t_url = a.get_attribute('href')
        ret_list.append({
            'title': a.text,
            'url': t_url
        })

    driver.quit()

    return ret_list

def move_to_minio(list):
    # 我们需要将所有竞赛的图片文件放到minio之中
    from minio import Minio
    from minio.error import S3Error

    client = Minio(
        MINIO_HOST,
        access_key=MINIO_ACC_KEY,
        secret_key=MINIO_SEC_KEY,
        secure=False
    )

    found = client.bucket_exists("geexin-news-pics")
    if not found:
        client.make_bucket("geexin-news-pics")

    for item in list:
        client.fput_object(
            "geexin-news-pics", f'{item["img"]}', f'{IMG_BASE}/{item["img"]}',
        )


def get_updated_list(cate_id, total_list):
    # 我们要获得新的list, 这样才知道哪些是新的
    recorded_list = json.loads(get(f'{HOST}/news/?cate={cate_id}&&count={len(total_list)}', headers=HEAD).text)
    recorded_list = [{'title': x['title'], 'url': x['url']} for x in recorded_list]
    new_list = [x for x in total_list if x not in recorded_list] 
    return new_list

if __name__ == '__main__':
    print(f'#### Start fetching cate:{CATE}')
    list = get_info_list(url)
    list.reverse()

    list = get_updated_list(CATE, list)
    post_list = [
        {
            **a,
            'cate': CATE,
            'img': str(hash(a['url'])) + '.png',
            'xpath': xpath,
        } for a in list
    ]

    save_news_list(post_list)

    print(f'#### Start move to MINIO')
    move_to_minio(post_list)
    print(f'#### Finished move to MINIO')

    res = post(f'{HOST}/news/', headers=HEAD, json=post_list)
    if res.status_code == 201:
        print(f'#### Create success, inserted: {len(post_list)} records')
    else:
        print('#### Create error')
